{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "from tqdm import tqdm\n",
    "import lightgbm as lgb\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.model_selection import GridSearchCV  \n",
    "from sklearn import metrics\n",
    "import warnings\n",
    "import matplotlib.pyplot as plt\n",
    "from datetime import datetime\n",
    "\n",
    "pd.set_option('display.max_columns', 100)\n",
    "pd.set_option('display.max_rows', 200)\n",
    "warnings.filterwarnings('ignore')\n",
    "%matplotlib inline\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save_data \n",
    "def read_csv(path):\n",
    "    df = pd.read_csv(path)\n",
    "    df = df.iloc[::-1]\n",
    "df_1 = pd.DataFrame()\n",
    "train_path ='hy_round1_train_20200102/hy_round1_train_20200102/'\n",
    "test_path ='hy_round1_testA_20200102/hy_round1_testA_20200102/'\n",
    "train_n = len(os.listdir(train_path))\n",
    "test_n = len(os.listdir(test_path))\n",
    "\n",
    "\n",
    "def get_df(path):\n",
    "    df_1 = pd.DataFrame()\n",
    "    n = len(os.listdir(path))\n",
    "    for i in range(n):\n",
    "        if path == 'hy_round1_testA_20200102/hy_round1_testA_20200102/':\n",
    "            i = i+ 7000\n",
    "        df = pd.read_csv(path+'%s.csv' %i)\n",
    "        df = df.iloc[::-1]\n",
    "        df_1 = pd.concat([df_1,df], axis =0 ) \n",
    "    return df_1\n",
    "\n",
    "# train_df = get_df(train_path)\n",
    "# train_df.columns = ['ship','x','y','v','d','time','type']\n",
    "# train_df.to_hdf('../train_raw.h5', 'df', mode='w')\n",
    "# test_df = get_df(test_path)\n",
    "# test_df.columns = ['ship','x','y','v','d','time']\n",
    "# test_df.to_hdf('../test_raw.h5', 'df', mode='w')\n",
    "train_df = pd.read_hdf('../train_raw.h5')\n",
    "test_df = pd.read_hdf('../test_raw.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ship</th>\n",
       "      <th>x</th>\n",
       "      <th>y</th>\n",
       "      <th>v</th>\n",
       "      <th>d</th>\n",
       "      <th>time</th>\n",
       "      <th>type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>413</th>\n",
       "      <td>0</td>\n",
       "      <td>6.118352e+06</td>\n",
       "      <td>5.130672e+06</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1107 12:09:28</td>\n",
       "      <td>拖网</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>412</th>\n",
       "      <td>0</td>\n",
       "      <td>6.118352e+06</td>\n",
       "      <td>5.130672e+06</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1107 12:18:30</td>\n",
       "      <td>拖网</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>411</th>\n",
       "      <td>0</td>\n",
       "      <td>6.118352e+06</td>\n",
       "      <td>5.130672e+06</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1107 12:28:32</td>\n",
       "      <td>拖网</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>410</th>\n",
       "      <td>0</td>\n",
       "      <td>6.118352e+06</td>\n",
       "      <td>5.130672e+06</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1107 12:38:32</td>\n",
       "      <td>拖网</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>409</th>\n",
       "      <td>0</td>\n",
       "      <td>6.118352e+06</td>\n",
       "      <td>5.130672e+06</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1107 12:48:30</td>\n",
       "      <td>拖网</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     ship             x             y    v  d           time type\n",
       "413     0  6.118352e+06  5.130672e+06  0.0  0  1107 12:09:28   拖网\n",
       "412     0  6.118352e+06  5.130672e+06  0.0  0  1107 12:18:30   拖网\n",
       "411     0  6.118352e+06  5.130672e+06  0.0  0  1107 12:28:32   拖网\n",
       "410     0  6.118352e+06  5.130672e+06  0.0  0  1107 12:38:32   拖网\n",
       "409     0  6.118352e+06  5.130672e+06  0.0  0  1107 12:48:30   拖网"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for v  / d \n",
    "def cut_feature2(df):\n",
    "    # cut v\n",
    "    v_conf0 = df['v'] > 0\n",
    "    v_conf1 = df['v'] >= 1\n",
    "    v_conf2 = df['v'] >= 2\n",
    "    v_conf3 = df['v'] >= 6\n",
    "#     v_conf4 = df['v'] >= 10\n",
    "    df['v_cut'] = 100\n",
    "    df['v_cut'] = np.where(v_conf0, 0, df['v_cut'])\n",
    "    df['v_cut'] = np.where(v_conf1, 1, df['v_cut'])\n",
    "    df['v_cut'] = np.where(v_conf2, 2, df['v_cut'])\n",
    "    df['v_cut'] = np.where(v_conf3, 3, df['v_cut'])\n",
    "#     df['v_cut'] = np.where(v_conf4, 4, df['v_cut'])\n",
    "    \n",
    "    # cut d\n",
    "    d_conf1 = df['d'] >= 50\n",
    "    d_conf2 = df['d'] >= 150\n",
    "    d_conf3 = df['d'] >= 250\n",
    "    \n",
    "    df['d_cut'] = 0\n",
    "    df['d_cut'] = np.where(d_conf1, 1, df['d_cut'])\n",
    "    df['d_cut'] = np.where(d_conf2, 2, df['d_cut'])\n",
    "    df['d_cut'] = np.where(d_conf3, 3, df['d_cut'])\n",
    "    \n",
    "    return df\n",
    "\n",
    "# train_df_cut2 = cut_feature2(train_df)   \n",
    "def extract_dt(df):\n",
    "    df['time'] = pd.to_datetime(df['time'], format='%m%d %H:%M:%S')\n",
    "    # df['month'] = df['time'].dt.month\n",
    "    # df['day'] = df['time'].dt.day\n",
    "    df['date'] = df['time'].dt.date\n",
    "    df['hour'] = df['time'].dt.hour\n",
    "    # df = df.drop_duplicates(['ship','month'])\n",
    "    df['weekday'] = df['time'].dt.weekday\n",
    "    return df\n",
    "\n",
    "def hour_feature(df):\n",
    "    \n",
    "    conf1 = df['hour'] >= 6\n",
    "#     conf2 = df['hour'] >= 12\n",
    "    conf2 = df['hour'] <= 18\n",
    "    \n",
    "    df['hour_cut'] = 0 #night\n",
    "    df['hour_cut'] = np.where(conf1 & conf2, 1, df['hour_cut']) # day\n",
    "#     df['hour_cut'] = np.where(conf2, 2, df['hour_cut']) # p.m.\n",
    "#     df['hour_cut'] = np.where(conf3, 3, df['hour_cut']) # night\n",
    "    \n",
    "    return df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df1 = extract_dt(train_df)\n",
    "test_df1 = extract_dt(test_df)\n",
    "\n",
    "train_df_cut2 = cut_feature2(train_df1)\n",
    "test_df_cut2 = cut_feature2(test_df1)\n",
    "\n",
    "train_df_cut2 = hour_feature(train_df_cut2)\n",
    "test_df_cut2 = hour_feature(test_df_cut2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "## train data feature \n",
    "train_df_cut2 = cut_feature2(train_df)     \n",
    "# train_df_cut2.head()\n",
    "train_df_cut_groupv2 = train_df_cut2.groupby(['ship', 'v_cut'])['x'].count().reset_index().sort_values(by =['ship','v_cut'])\n",
    "train_df_cut_groupvd2 = train_df_cut2.groupby(['ship','v_cut','d_cut'])['x'].count().reset_index().sort_values(by =['ship','v_cut','d_cut'])\n",
    "# print(train_df_cut_groupv2.tail(2))\n",
    "# print(train_df_cut_groupvd2.head(2))\n",
    "\n",
    "train_df_group_n2 = train_df_cut_groupv2.merge(train_df_cut_groupvd2, how = 'left',on =['ship','v_cut'],suffixes=('','_n'))\n",
    "train_df_group_n2['v_d_rate'] = train_df_group_n2['x_n']/train_df_group_n2['x']\n",
    "train_df_group_n2['v_d_cut'] = train_df_group_n2['v_cut'].astype('str')+'_'+ train_df_group_n2['d_cut'].astype('str')\n",
    "\n",
    "# train_df_group_rate2 = v_d_cut_feat(train_df_group_n2, 6, 4)\n",
    "# # 保存到本地\n",
    "# train_df_group_rate2.to_hdf('../train_df_group_rate2.h5' , 'df', mode='w')\n",
    "train_df_group_rate2 = pd.read_hdf('../train_df_group_rate2.h5')\n",
    "\n",
    "train_df_cut_group1 = train_df.groupby(['ship'])['x'].count().reset_index().sort_values(by =['ship'])\n",
    "train_df_cut_groupv3 = train_df_cut_groupv2.merge(train_df_cut_group1,how ='left', on ='ship',suffixes=('','_cnt'))\n",
    "train_df_cut_groupv3['v_cut_rate'] = train_df_cut_groupv3['x']/train_df_cut_groupv3['x_cnt']\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ship</th>\n",
       "      <th>v_cut</th>\n",
       "      <th>x</th>\n",
       "      <th>x_cnt</th>\n",
       "      <th>v_cut_rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>26</td>\n",
       "      <td>414</td>\n",
       "      <td>0.062802</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>414</td>\n",
       "      <td>0.002415</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>414</td>\n",
       "      <td>0.019324</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>414</td>\n",
       "      <td>0.024155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>100</td>\n",
       "      <td>369</td>\n",
       "      <td>414</td>\n",
       "      <td>0.891304</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ship  v_cut    x  x_cnt  v_cut_rate\n",
       "0     0      0   26    414    0.062802\n",
       "1     0      1    1    414    0.002415\n",
       "2     0      2    8    414    0.019324\n",
       "3     0      3   10    414    0.024155\n",
       "4     0    100  369    414    0.891304"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df_cut_groupv3.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# v_cut_rate\n",
    "def v_cut_rate_feat(df, n ,test_mode = False):\n",
    "    df3 = pd.DataFrame()\n",
    "    for id in range(df['ship'].nunique()):\n",
    "#     for id in range(3):\n",
    "        if test_mode :\n",
    "            id = id +7000\n",
    "        print(id)\n",
    "        df1 = pd.DataFrame({'ship': id}, index = [id]) \n",
    "        for i in range(n):\n",
    "            i = i+1\n",
    "            if not df.loc[(df['v_cut']== i) & (df['ship']== id),'v_cut_rate'].empty : \n",
    "                aa = df.loc[(df['v_cut']== i) & (df['ship']== id),'v_cut_rate'].reset_index()\n",
    "                dict1 = {'v_cut_'+ str(i)+'_rate': aa['v_cut_rate'][0]}                \n",
    "            else:\n",
    "                dict1 = {'v_cut_'+ str(i)+'_rate': 0}\n",
    "            df2 = pd.DataFrame(dict1, index = [id])\n",
    "            df1 = pd.concat([df1,df2], axis =1 )\n",
    "        df3 = pd.concat([df3,df1], axis =0 )\n",
    "    return df3    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df_group_vcnt_rate1 =pd.read_hdf('../train_df_group_vcnt_2_rate1.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "## test data feature \n",
    "test_df_cut2 = cut_feature2(test_df)     \n",
    "# test_df_cut2.head()\n",
    "test_df_cut_groupv2 = test_df_cut2.groupby(['ship', 'v_cut'])['x'].count().reset_index().sort_values(by =['ship','v_cut'])\n",
    "test_df_cut_groupvd2 = test_df_cut2.groupby(['ship','v_cut','d_cut'])['x'].count().reset_index().sort_values(by =['ship','v_cut','d_cut'])\n",
    "# print(test_df_cut_groupv2.tail(2))\n",
    "# print(test_df_cut_groupvd2.head(2))\n",
    "\n",
    "test_df_group_n2 = test_df_cut_groupv2.merge(test_df_cut_groupvd2, how = 'left',on =['ship','v_cut'],suffixes=('','_n'))\n",
    "test_df_group_n2['v_d_rate'] = test_df_group_n2['x_n']/test_df_group_n2['x']\n",
    "test_df_group_n2['v_d_cut'] = test_df_group_n2['v_cut'].astype('str')+'_'+ test_df_group_n2['d_cut'].astype('str')\n",
    "\n",
    "# test_df_group_rate2 = v_d_cut_feat(test_df_group_n2, 6, 4)\n",
    "# # 保存到本地\n",
    "# test_df_group_rate2.to_hdf('../test_df_group_rate2.h5' , 'df', mode='w')\n",
    "# test_df_group_rate2 = pd.read_hdf('../test_df_group_rate2.h5')\n",
    "\n",
    "test_df_cut_group1 = test_df.groupby(['ship'])['x'].count().reset_index().sort_values(by =['ship'])\n",
    "test_df_cut_groupv3 = test_df_cut_groupv2.merge(test_df_cut_group1,how ='left', on ='ship',suffixes=('','_cnt'))\n",
    "test_df_cut_groupv3['v_cut_rate'] = test_df_cut_groupv3['x']/test_df_cut_groupv3['x_cnt']\n",
    "\n",
    "# test_df_group_vcnt_rate1 = v_cut_rate_feat(test_df_cut_groupv3, 2 ,test_mode = True)\n",
    "# test_df_group_vcnt_rate1.to_hdf('../test_df_group_vcnt_2_rate1.h5' , 'df', mode='w')\n",
    "test_df_group_vcnt_rate1 = pd.read_hdf('../test_df_group_vcnt_2_rate1.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ship</th>\n",
       "      <th>v_cut_1_rate</th>\n",
       "      <th>v_cut_2_rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.002415</td>\n",
       "      <td>0.019324</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.005195</td>\n",
       "      <td>0.301299</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0.017167</td>\n",
       "      <td>0.038627</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0.044776</td>\n",
       "      <td>0.194030</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0.039900</td>\n",
       "      <td>0.089776</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ship  v_cut_1_rate  v_cut_2_rate\n",
       "0     0      0.002415      0.019324\n",
       "1     1      0.005195      0.301299\n",
       "2     2      0.017167      0.038627\n",
       "3     3      0.044776      0.194030\n",
       "4     4      0.039900      0.089776"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df_group_vcnt_rate1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# d_cut_rate\n",
    "def d_cut_rate_feat(df, n ,test_mode = False):\n",
    "    df3 = pd.DataFrame()\n",
    "    for id in range(df['ship'].nunique()):\n",
    "#     for id in range(3):\n",
    "        if test_mode :\n",
    "            id = id +7000\n",
    "        print(id)\n",
    "        df1 = pd.DataFrame({'ship': id}, index = [id]) \n",
    "        for i in range(n):\n",
    "            if not df.loc[(df['d_cut']== i) & (df['ship']== id),'d_cut_rate'].empty : \n",
    "                aa = df.loc[(df['d_cut']== i) & (df['ship']== id),'d_cut_rate'].reset_index()\n",
    "                dict1 = {'d_cut_'+ str(i)+'_rate': aa['d_cut_rate'][0]}                \n",
    "            else:\n",
    "                dict1 = {'d_cut_'+ str(i)+'_rate': 0}\n",
    "            df2 = pd.DataFrame(dict1, index = [id])\n",
    "            df1 = pd.concat([df1,df2], axis =1 )\n",
    "        df3 = pd.concat([df3,df1], axis =0 )\n",
    "    return df3   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# d_cut rate\n",
    "train_df_cut_group_d3 = train_df_cut_groupvd2.groupby(['ship','d_cut'])['x'].sum().reset_index()\n",
    "train_df_cut_group_d4 = train_df_cut_group_d3.merge(train_df_cut_group1,how ='left', on ='ship',suffixes=('','_cnt'))\n",
    "train_df_cut_group_d4['d_cut_rate'] = train_df_cut_group_d4['x']/train_df_cut_group_d4['x_cnt']\n",
    "\n",
    "# train v_cut_rate\n",
    "# train_df_group_dcnt_rate1 = d_cut_rate_feat(train_df_cut_group_d4, train_df_cut_group_d4['d_cut'].nunique() ,False)\n",
    "# train_df_group_dcnt_rate1.to_hdf('../train_df_group_dcnt_rate1.h5' , 'df', mode='w')\n",
    "train_df_group_dcnt_rate1 = pd.read_hdf('../train_df_group_dcnt_rate1.h5')\n",
    "# train_df_group_dcnt_rate1.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ship</th>\n",
       "      <th>d_cut_0_rate</th>\n",
       "      <th>d_cut_1_rate</th>\n",
       "      <th>d_cut_2_rate</th>\n",
       "      <th>d_cut_3_rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.954106</td>\n",
       "      <td>0.045894</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.685714</td>\n",
       "      <td>0.127273</td>\n",
       "      <td>0.132468</td>\n",
       "      <td>0.054545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0.399142</td>\n",
       "      <td>0.248927</td>\n",
       "      <td>0.124464</td>\n",
       "      <td>0.227468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0.414925</td>\n",
       "      <td>0.217910</td>\n",
       "      <td>0.107463</td>\n",
       "      <td>0.259701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0.336658</td>\n",
       "      <td>0.231920</td>\n",
       "      <td>0.197007</td>\n",
       "      <td>0.234414</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ship  d_cut_0_rate  d_cut_1_rate  d_cut_2_rate  d_cut_3_rate\n",
       "0     0      0.954106      0.045894      0.000000      0.000000\n",
       "1     1      0.685714      0.127273      0.132468      0.054545\n",
       "2     2      0.399142      0.248927      0.124464      0.227468\n",
       "3     3      0.414925      0.217910      0.107463      0.259701\n",
       "4     4      0.336658      0.231920      0.197007      0.234414"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df_group_dcnt_rate1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test d_cut_rate\n",
    "test_df_cut_group_d3 = test_df_cut_groupvd2.groupby(['ship','d_cut'])['x'].sum().reset_index()\n",
    "test_df_cut_group_d4 = test_df_cut_group_d3.merge(test_df_cut_group1,how ='left', on ='ship',suffixes=('','_cnt'))\n",
    "test_df_cut_group_d4['d_cut_rate'] = test_df_cut_group_d4['x']/test_df_cut_group_d4['x_cnt']\n",
    "\n",
    "# test_df_group_dcnt_rate1 = d_cut_rate_feat(test_df_cut_group_d4, test_df_cut_group_d4['d_cut'].nunique() ,True)\n",
    "# test_df_group_dcnt_rate1.to_hdf('../test_df_group_dcnt_rate1.h5' , 'df', mode='w')\n",
    "test_df_group_dcnt_rate1 = pd.read_hdf('../test_df_group_dcnt_rate1.h5')\n",
    "# test_df_group_dcnt_rate1.tail()\n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def v_hour_cut_feat(df, n, m ,test_mode = False):\n",
    "    df3 = pd.DataFrame()\n",
    "    for id in range(df['ship'].nunique()):\n",
    "        if test_mode:\n",
    "            id = id +7000\n",
    "        print(id)\n",
    "        df1 = pd.DataFrame({'ship': id}, index = [id])     \n",
    "        df2 = pd.DataFrame()\n",
    "        for i in range(n):  \n",
    "            i = i +1\n",
    "            for j in range(m):\n",
    "                a = str(i)+'_'+str(j)\n",
    "                if not df.loc[(df['v_hour_cut']== a) & (df['ship']== id),'v_hour_rate'].empty :\n",
    "                    aa = df.loc[(df['v_hour_cut']== a) & (df['ship']== id),'v_hour_rate'].reset_index()\n",
    "                    dict1 = {'v_hour_'+ str(i)+'_'+str(j): aa['v_hour_rate'][0]}\n",
    "                else:\n",
    "                    dict1 = {'v_hour_'+ str(i)+'_'+str(j): 0}\n",
    "                    \n",
    "                df2 = pd.DataFrame(dict1, index = [id])\n",
    "                df1 = pd.concat([df1,df2], axis =1 )\n",
    "        df3 = pd.concat([df3,df1], axis =0 )\n",
    "    return df3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train \n",
    "train_df_cut2 = extract_dt(train_df_cut2)\n",
    "train_df_cut2 = hour_feature(train_df_cut2)\n",
    "train_df_cut_groupvh = train_df_cut2.groupby(['ship','v_cut','hour_cut'])['x'].count().reset_index().sort_values(by =['ship','v_cut','hour_cut'])\n",
    "train_df_group_n3 = train_df_cut_groupv2.merge(train_df_cut_groupvh, how = 'left',on =['ship','v_cut'],suffixes=('','_n'))\n",
    "train_df_group_n3['v_hour_rate'] = train_df_group_n3['x_n']/train_df_group_n3['x']\n",
    "train_df_group_n3['v_hour_cut'] = train_df_group_n3['v_cut'].astype('str')+'_'+ train_df_group_n3['hour_cut'].astype('str')\n",
    "\n",
    "# train_df_group_vhour_rate3 = v_hour_cut_feat(train_df_group_n3, train_df_group_n3['v_cut'].nunique()-3, train_df_group_n3['hour_cut'].nunique(),test_mode = False)\n",
    "# 保存到本地\n",
    "# train_df_group_vhour_rate3.to_hdf('../train_df_group_v_hour2_rate.h5' , 'df', mode='w')\n",
    "train_df_group_vhour_rate3 = pd.read_hdf('../train_df_group_v_hour2_rate.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ship</th>\n",
       "      <th>v_hour_1_0</th>\n",
       "      <th>v_hour_1_1</th>\n",
       "      <th>v_hour_2_0</th>\n",
       "      <th>v_hour_2_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.508621</td>\n",
       "      <td>0.491379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0.500</td>\n",
       "      <td>0.500</td>\n",
       "      <td>0.555556</td>\n",
       "      <td>0.444444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0.600</td>\n",
       "      <td>0.400</td>\n",
       "      <td>0.723077</td>\n",
       "      <td>0.276923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0.625</td>\n",
       "      <td>0.375</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ship  v_hour_1_0  v_hour_1_1  v_hour_2_0  v_hour_2_1\n",
       "0     0       0.000       1.000    0.000000    1.000000\n",
       "1     1       0.000       1.000    0.508621    0.491379\n",
       "2     2       0.500       0.500    0.555556    0.444444\n",
       "3     3       0.600       0.400    0.723077    0.276923\n",
       "4     4       0.625       0.375    0.500000    0.500000"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df_group_vhour_rate3.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test \n",
    "test_df_cut2 = extract_dt(test_df_cut2)\n",
    "test_df_cut2 = hour_feature(test_df_cut2)\n",
    "test_df_cut_groupvh = test_df_cut2.groupby(['ship','v_cut','hour_cut'])['x'].count().reset_index().sort_values(by =['ship','v_cut','hour_cut'])\n",
    "test_df_group_n3 = test_df_cut_groupv2.merge(test_df_cut_groupvh, how = 'left',on =['ship','v_cut'],suffixes=('','_n'))\n",
    "test_df_group_n3['v_hour_rate'] = test_df_group_n3['x_n']/test_df_group_n3['x']\n",
    "test_df_group_n3['v_hour_cut'] = test_df_group_n3['v_cut'].astype('str')+'_'+ test_df_group_n3['hour_cut'].astype('str')\n",
    "\n",
    "# test_df_group_vhour_rate3 = v_hour_cut_feat(test_df_group_n3, test_df_group_n3['v_cut'].nunique()-3, test_df_group_n3['hour_cut'].nunique(),test_mode = True)\n",
    "# # 保存到本地\n",
    "# test_df_group_vhour_rate3.to_hdf('../test_df_group_v_hour_rate.h5' , 'df', mode='w')\n",
    "test_df_group_vhour_rate3 = pd.read_hdf('../test_df_group_v_hour_rate.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ship</th>\n",
       "      <th>v_hour_1_0</th>\n",
       "      <th>v_hour_1_1</th>\n",
       "      <th>v_hour_2_0</th>\n",
       "      <th>v_hour_2_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7000</th>\n",
       "      <td>7000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7001</th>\n",
       "      <td>7001</td>\n",
       "      <td>0.482759</td>\n",
       "      <td>0.517241</td>\n",
       "      <td>0.421642</td>\n",
       "      <td>0.578358</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7002</th>\n",
       "      <td>7002</td>\n",
       "      <td>0.352941</td>\n",
       "      <td>0.647059</td>\n",
       "      <td>0.506944</td>\n",
       "      <td>0.493056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7003</th>\n",
       "      <td>7003</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.555556</td>\n",
       "      <td>0.444444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7004</th>\n",
       "      <td>7004</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.153846</td>\n",
       "      <td>0.846154</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ship  v_hour_1_0  v_hour_1_1  v_hour_2_0  v_hour_2_1\n",
       "7000  7000    0.500000    0.500000    0.333333    0.666667\n",
       "7001  7001    0.482759    0.517241    0.421642    0.578358\n",
       "7002  7002    0.352941    0.647059    0.506944    0.493056\n",
       "7003  7003    0.000000    1.000000    0.555556    0.444444\n",
       "7004  7004    0.250000    0.750000    0.153846    0.846154"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df_group_vhour_rate3.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def group_feature(df, key, target, aggs):   \n",
    "    agg_dict = {}\n",
    "    for ag in aggs:\n",
    "        agg_dict[f'{target}_{ag}'] = ag\n",
    "    print(agg_dict)\n",
    "    t = df.groupby(key)[target].agg(agg_dict).reset_index()\n",
    "    return t\n",
    "\n",
    "\n",
    "\n",
    "def extract_feature(df, train):\n",
    "    \n",
    "    t = group_feature(df, 'ship','x',['max','min','mean','std','skew'])\n",
    "    train = pd.merge(train, t, on='ship', how='left')\n",
    "    t = group_feature(df, 'ship','x',['count'])\n",
    "    train = pd.merge(train, t, on='ship', how='left')\n",
    "    t = group_feature(df, 'ship','y',['max','min','mean','std','skew'])\n",
    "    train = pd.merge(train, t, on='ship', how='left')\n",
    "    t = group_feature(df, 'ship','v',['max','min','mean','std','skew'])\n",
    "    train = pd.merge(train, t, on='ship', how='left')\n",
    "    t = group_feature(df, 'ship','d',['max','min','mean','std','skew'])\n",
    "    train = pd.merge(train, t, on='ship', how='left')\n",
    "\n",
    "#     t = df.groupby(['ship'])['x'].cov(df['y']).reset_index()\n",
    "#     train = pd.merge(train, t, on='ship', how='left')\n",
    "    \n",
    "    \n",
    "    train['x_max_x_min'] = train['x_max'] - train['x_min']\n",
    "    train['y_max_y_min'] = train['y_max'] - train['y_min']\n",
    "    train['y_max_x_min'] = train['y_max'] - train['x_min']\n",
    "    train['x_max_y_min'] = train['x_max'] - train['y_min']\n",
    "    train['slope'] = train['y_max_y_min'] / np.where(train['x_max_x_min']==0, 0.001, train['x_max_x_min'])\n",
    "    train['area'] = train['x_max_x_min'] * train['y_max_y_min']\n",
    "    \n",
    "#     train['dis_mean'] = np.sqrt((5000249.625693836 - train['x_mean'])**2 + (3784749- train['y_mean'])**2 )  \n",
    "#     train['dis_x_mean'] = 5000249.625693836 - train['x_mean'] \n",
    "#     train['dis_y_mean'] = 3784749 - train['y_mean']    \n",
    "    train['dis_mean'] = np.sqrt((6403859.289610631 - train['x_mean'])**2 + (5381546.9315851545- train['y_mean'])**2 )  \n",
    "    train['dis_x_mean'] = 6403859.289610631 - train['x_mean'] \n",
    "    train['dis_y_mean'] = 5381546.9315851545- train['y_mean']\n",
    "    \n",
    "    mode_hour = df.groupby('ship')['hour'].agg(lambda x:x.value_counts().index[0]).to_dict()\n",
    "    train['mode_hour'] = train['ship'].map(mode_hour)\n",
    "    \n",
    "####add \n",
    "#     for col in ['x', 'y', 'v','d']:\n",
    "#         df[col + '_diff1'] = df.groupby(['ship'])[col].diff(1)\n",
    "        \n",
    "# #     for col in ['x', 'y', 'v','d']:\n",
    "# #         df[col + '_diff_2'] = df.groupby(['ship'])[col].diff(2)         \n",
    "#     for col in ['x', 'y', 'v','d']:\n",
    "#         df[col + '_diff3'] = df.groupby(['ship'])[col].diff(3) \n",
    "        \n",
    "#     for col in ['x', 'y', 'v','d']:\n",
    "#         for diff in ['_diff1']:\n",
    "# #         for diff in ['_diff1','_diff3']:\n",
    "#             t = group_feature(df, 'ship',col+diff ,['std','skew'])\n",
    "#             train = pd.merge(train, t, on='ship', how='left')  \n",
    "#     train['y_max_x_min_diff1'] = train['y_diff1_max'] - train['x_diff1_min']\n",
    "#     train['y_max_x_min_diff3'] = train['y_diff3_max'] - train['x_diff3_min']\n",
    "                      \n",
    "        \n",
    "    \n",
    "    for col in ['x', 'y', 'time']:\n",
    "        df[col + '_diff'] = df.groupby(['ship'])[col].diff()\n",
    "    df['time_diff'] = df['time_diff'].values/np.timedelta64(1, 'h')\n",
    "    df['ind_dist'] = np.sqrt(df['x_diff']**2  + df['y_diff']**2)\n",
    "    df['ind_speed'] = df['ind_dist'] / df['time_diff']\n",
    "    df.fillna(0, inplace=True)\n",
    "    tot_distance = df.groupby(['ship'])['ind_dist'].sum().rename('tot_distance').reset_index()\n",
    "    avg_speed = df.groupby(['ship'])['ind_speed'].mean().rename('avg_speed').reset_index()\n",
    "\n",
    "    train = train.merge(tot_distance, on=['ship'], how='left')\n",
    "    train = train.merge(avg_speed, on=['ship'], how='left')   \n",
    "    \n",
    "#     t = group_feature(df, 'ship','hour',['max','min'])\n",
    "#     train = pd.merge(train, t, on='ship', how='left')\n",
    "    \n",
    "#     hour_nunique = df.groupby('ship')['hour'].nunique().to_dict()\n",
    "#     date_nunique = df.groupby('ship')['date'].nunique().to_dict()\n",
    "#     train['hour_nunique'] = train['ship'].map(hour_nunique)\n",
    "#     train['date_nunique'] = train['ship'].map(date_nunique)\n",
    "\n",
    "#     t = df.groupby('ship')['time'].agg({'diff_time':lambda x:np.max(x)-np.min(x)}).reset_index()\n",
    "#     t['diff_day'] = t['diff_time'].dt.days\n",
    "#     t['diff_second'] = t['diff_time'].dt.seconds\n",
    "#     train = pd.merge(train, t, on='ship', how='left')\n",
    "    return train\n",
    "\n",
    "def extract_dt(df):\n",
    "    df['time'] = pd.to_datetime(df['time'], format='%m%d %H:%M:%S')\n",
    "    # df['month'] = df['time'].dt.month\n",
    "    # df['day'] = df['time'].dt.day\n",
    "    df['date'] = df['time'].dt.date\n",
    "    df['hour'] = df['time'].dt.hour\n",
    "    # df = df.drop_duplicates(['ship','month'])\n",
    "    df['weekday'] = df['time'].dt.weekday\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_label = train_df.drop_duplicates('ship')\n",
    "test_label = test_df.drop_duplicates('ship')\n",
    "\n",
    "type_map = dict(zip(train_label['type'].unique(), np.arange(3)))\n",
    "type_map_rev = {v:k for k,v in type_map.items()}\n",
    "train_label['type'] = train_label['type'].map(type_map)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'x_max': 'max', 'x_min': 'min', 'x_mean': 'mean', 'x_std': 'std', 'x_skew': 'skew'}\n",
      "{'x_count': 'count'}\n",
      "{'y_max': 'max', 'y_min': 'min', 'y_mean': 'mean', 'y_std': 'std', 'y_skew': 'skew'}\n",
      "{'v_max': 'max', 'v_min': 'min', 'v_mean': 'mean', 'v_std': 'std', 'v_skew': 'skew'}\n",
      "{'d_max': 'max', 'd_min': 'min', 'd_mean': 'mean', 'd_std': 'std', 'd_skew': 'skew'}\n"
     ]
    }
   ],
   "source": [
    "train_label = extract_feature(train_df, train_label)\n",
    "# test_label = extract_feature(test_df, test_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'x_max': 'max', 'x_min': 'min', 'x_mean': 'mean', 'x_std': 'std', 'x_skew': 'skew'}\n",
      "{'x_count': 'count'}\n",
      "{'y_max': 'max', 'y_min': 'min', 'y_mean': 'mean', 'y_std': 'std', 'y_skew': 'skew'}\n",
      "{'v_max': 'max', 'v_min': 'min', 'v_mean': 'mean', 'v_std': 'std', 'v_skew': 'skew'}\n",
      "{'d_max': 'max', 'd_min': 'min', 'd_mean': 'mean', 'd_std': 'std', 'd_skew': 'skew'}\n"
     ]
    }
   ],
   "source": [
    "test_label = extract_feature(test_df, test_label)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def merge_feature(df_v_cut, df_v_hour, df_label, df_d_cut):\n",
    "    df = df_label.merge(df_v_cut, how = 'left', on ='ship')\n",
    "#     df = df.merge(df_v_d, how = 'left', on ='ship')|\n",
    "    df = df.merge(df_v_hour, how = 'left', on ='ship')\n",
    "    df = df.merge(df_d_cut, how = 'left', on ='ship')\n",
    "    \n",
    "    return df "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_features = merge_feature(train_df_group_vcnt_rate1, train_df_group_vhour_rate3, train_label,train_df_group_dcnt_rate1)\n",
    "test_features = merge_feature(test_df_group_vcnt_rate1, test_df_group_vhour_rate3, test_label, test_df_group_dcnt_rate1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ship</th>\n",
       "      <th>x</th>\n",
       "      <th>y_x</th>\n",
       "      <th>v_x</th>\n",
       "      <th>d_x</th>\n",
       "      <th>time</th>\n",
       "      <th>type</th>\n",
       "      <th>date</th>\n",
       "      <th>hour</th>\n",
       "      <th>weekday</th>\n",
       "      <th>v_cut</th>\n",
       "      <th>d_cut</th>\n",
       "      <th>hour_cut</th>\n",
       "      <th>x_max</th>\n",
       "      <th>x_min</th>\n",
       "      <th>x_mean</th>\n",
       "      <th>x_std</th>\n",
       "      <th>x_skew</th>\n",
       "      <th>x_count</th>\n",
       "      <th>y_max</th>\n",
       "      <th>y_min</th>\n",
       "      <th>y_mean</th>\n",
       "      <th>y_std</th>\n",
       "      <th>y_skew</th>\n",
       "      <th>v_max</th>\n",
       "      <th>v_min</th>\n",
       "      <th>v_mean</th>\n",
       "      <th>v_std</th>\n",
       "      <th>v_skew</th>\n",
       "      <th>d_max</th>\n",
       "      <th>d_min</th>\n",
       "      <th>d_mean</th>\n",
       "      <th>d_std</th>\n",
       "      <th>d_skew</th>\n",
       "      <th>x_q25</th>\n",
       "      <th>y_y</th>\n",
       "      <th>v_y</th>\n",
       "      <th>d_y</th>\n",
       "      <th>x_q75</th>\n",
       "      <th>y</th>\n",
       "      <th>v</th>\n",
       "      <th>x_max_x_min</th>\n",
       "      <th>y_max_y_min</th>\n",
       "      <th>y_max_x_min</th>\n",
       "      <th>x_max_y_min</th>\n",
       "      <th>slope</th>\n",
       "      <th>area</th>\n",
       "      <th>dis_mean</th>\n",
       "      <th>mode_hour</th>\n",
       "      <th>tot_distance</th>\n",
       "      <th>avg_speed</th>\n",
       "      <th>v_cut_1_rate</th>\n",
       "      <th>v_cut_2_rate</th>\n",
       "      <th>v_hour_1_0</th>\n",
       "      <th>v_hour_1_1</th>\n",
       "      <th>v_hour_2_0</th>\n",
       "      <th>v_hour_2_1</th>\n",
       "      <th>d_cut_0_rate</th>\n",
       "      <th>d_cut_1_rate</th>\n",
       "      <th>d_cut_2_rate</th>\n",
       "      <th>d_cut_3_rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>6.118352e+06</td>\n",
       "      <td>5.130672e+06</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>1900-11-07 12:09:28</td>\n",
       "      <td>0</td>\n",
       "      <td>1900-11-07</td>\n",
       "      <td>12</td>\n",
       "      <td>2</td>\n",
       "      <td>100</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>6.152038e+06</td>\n",
       "      <td>6.118352e+06</td>\n",
       "      <td>6.119351e+06</td>\n",
       "      <td>5037.320747</td>\n",
       "      <td>5.255558</td>\n",
       "      <td>414</td>\n",
       "      <td>5.130781e+06</td>\n",
       "      <td>5.124873e+06</td>\n",
       "      <td>5.130494e+06</td>\n",
       "      <td>850.264541</td>\n",
       "      <td>-4.762308</td>\n",
       "      <td>9.39</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.265966</td>\n",
       "      <td>1.321248</td>\n",
       "      <td>5.520205</td>\n",
       "      <td>129</td>\n",
       "      <td>0</td>\n",
       "      <td>4.613527</td>\n",
       "      <td>21.247770</td>\n",
       "      <td>4.483093</td>\n",
       "      <td>6.118352e+06</td>\n",
       "      <td>5.130672e+06</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.118352e+06</td>\n",
       "      <td>5.130672e+06</td>\n",
       "      <td>0.00</td>\n",
       "      <td>33686.667453</td>\n",
       "      <td>5907.975523</td>\n",
       "      <td>-987570.399385</td>\n",
       "      <td>1.027165e+06</td>\n",
       "      <td>0.175380</td>\n",
       "      <td>1.990200e+08</td>\n",
       "      <td>3.794369e+05</td>\n",
       "      <td>15</td>\n",
       "      <td>35968.022927</td>\n",
       "      <td>490.859961</td>\n",
       "      <td>0.002415</td>\n",
       "      <td>0.019324</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.954106</td>\n",
       "      <td>0.045894</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>6.102450e+06</td>\n",
       "      <td>5.112760e+06</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>1900-11-07 12:00:34</td>\n",
       "      <td>0</td>\n",
       "      <td>1900-11-07</td>\n",
       "      <td>12</td>\n",
       "      <td>2</td>\n",
       "      <td>100</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>6.102450e+06</td>\n",
       "      <td>6.049472e+06</td>\n",
       "      <td>6.091460e+06</td>\n",
       "      <td>16543.394419</td>\n",
       "      <td>-1.058454</td>\n",
       "      <td>385</td>\n",
       "      <td>5.112874e+06</td>\n",
       "      <td>5.042857e+06</td>\n",
       "      <td>5.094050e+06</td>\n",
       "      <td>26764.042729</td>\n",
       "      <td>-0.802446</td>\n",
       "      <td>10.47</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.607922</td>\n",
       "      <td>2.412688</td>\n",
       "      <td>1.590284</td>\n",
       "      <td>336</td>\n",
       "      <td>0</td>\n",
       "      <td>56.153247</td>\n",
       "      <td>91.449382</td>\n",
       "      <td>1.418867</td>\n",
       "      <td>6.074562e+06</td>\n",
       "      <td>5.061049e+06</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.102450e+06</td>\n",
       "      <td>5.112760e+06</td>\n",
       "      <td>3.45</td>\n",
       "      <td>52978.013345</td>\n",
       "      <td>70016.655842</td>\n",
       "      <td>-936597.872550</td>\n",
       "      <td>1.059593e+06</td>\n",
       "      <td>1.321617</td>\n",
       "      <td>3.709343e+09</td>\n",
       "      <td>4.245555e+05</td>\n",
       "      <td>19</td>\n",
       "      <td>190031.884257</td>\n",
       "      <td>2657.205033</td>\n",
       "      <td>0.005195</td>\n",
       "      <td>0.301299</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.508621</td>\n",
       "      <td>0.491379</td>\n",
       "      <td>0.685714</td>\n",
       "      <td>0.127273</td>\n",
       "      <td>0.132468</td>\n",
       "      <td>0.054545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>6.182482e+06</td>\n",
       "      <td>5.193696e+06</td>\n",
       "      <td>0.11</td>\n",
       "      <td>308</td>\n",
       "      <td>1900-11-14 12:07:01</td>\n",
       "      <td>0</td>\n",
       "      <td>1900-11-14</td>\n",
       "      <td>12</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>6.183191e+06</td>\n",
       "      <td>6.182482e+06</td>\n",
       "      <td>6.183011e+06</td>\n",
       "      <td>207.869601</td>\n",
       "      <td>-2.155218</td>\n",
       "      <td>233</td>\n",
       "      <td>5.193696e+06</td>\n",
       "      <td>5.193576e+06</td>\n",
       "      <td>5.193682e+06</td>\n",
       "      <td>21.740609</td>\n",
       "      <td>-4.563165</td>\n",
       "      <td>50.46</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.595150</td>\n",
       "      <td>3.415824</td>\n",
       "      <td>13.631590</td>\n",
       "      <td>360</td>\n",
       "      <td>0</td>\n",
       "      <td>123.356223</td>\n",
       "      <td>123.097127</td>\n",
       "      <td>0.657506</td>\n",
       "      <td>6.183090e+06</td>\n",
       "      <td>5.193685e+06</td>\n",
       "      <td>0.11</td>\n",
       "      <td>10.0</td>\n",
       "      <td>6.183090e+06</td>\n",
       "      <td>5.193685e+06</td>\n",
       "      <td>0.32</td>\n",
       "      <td>708.835147</td>\n",
       "      <td>120.565000</td>\n",
       "      <td>-988786.086021</td>\n",
       "      <td>9.896155e+05</td>\n",
       "      <td>0.170089</td>\n",
       "      <td>8.546071e+04</td>\n",
       "      <td>2.899437e+05</td>\n",
       "      <td>17</td>\n",
       "      <td>3384.780162</td>\n",
       "      <td>62.731755</td>\n",
       "      <td>0.017167</td>\n",
       "      <td>0.038627</td>\n",
       "      <td>0.500</td>\n",
       "      <td>0.500</td>\n",
       "      <td>0.555556</td>\n",
       "      <td>0.444444</td>\n",
       "      <td>0.399142</td>\n",
       "      <td>0.248927</td>\n",
       "      <td>0.124464</td>\n",
       "      <td>0.227468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>5.228590e+06</td>\n",
       "      <td>4.606725e+06</td>\n",
       "      <td>1.67</td>\n",
       "      <td>211</td>\n",
       "      <td>1900-11-07 12:15:11</td>\n",
       "      <td>0</td>\n",
       "      <td>1900-11-07</td>\n",
       "      <td>12</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>5.287805e+06</td>\n",
       "      <td>5.228590e+06</td>\n",
       "      <td>5.239159e+06</td>\n",
       "      <td>17503.714347</td>\n",
       "      <td>1.608637</td>\n",
       "      <td>335</td>\n",
       "      <td>4.608628e+06</td>\n",
       "      <td>4.577467e+06</td>\n",
       "      <td>4.601532e+06</td>\n",
       "      <td>11590.605179</td>\n",
       "      <td>-1.194210</td>\n",
       "      <td>10.09</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.471343</td>\n",
       "      <td>2.528593</td>\n",
       "      <td>2.135446</td>\n",
       "      <td>352</td>\n",
       "      <td>0</td>\n",
       "      <td>121.134328</td>\n",
       "      <td>121.758165</td>\n",
       "      <td>0.469794</td>\n",
       "      <td>5.229843e+06</td>\n",
       "      <td>4.595101e+06</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.236628e+06</td>\n",
       "      <td>4.608404e+06</td>\n",
       "      <td>2.21</td>\n",
       "      <td>59214.738740</td>\n",
       "      <td>31160.661097</td>\n",
       "      <td>-619962.107144</td>\n",
       "      <td>7.103375e+05</td>\n",
       "      <td>0.526232</td>\n",
       "      <td>1.845170e+09</td>\n",
       "      <td>1.401766e+06</td>\n",
       "      <td>22</td>\n",
       "      <td>153492.071398</td>\n",
       "      <td>2641.442772</td>\n",
       "      <td>0.044776</td>\n",
       "      <td>0.194030</td>\n",
       "      <td>0.600</td>\n",
       "      <td>0.400</td>\n",
       "      <td>0.723077</td>\n",
       "      <td>0.276923</td>\n",
       "      <td>0.414925</td>\n",
       "      <td>0.217910</td>\n",
       "      <td>0.107463</td>\n",
       "      <td>0.259701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>7.065534e+06</td>\n",
       "      <td>6.116614e+06</td>\n",
       "      <td>0.00</td>\n",
       "      <td>127</td>\n",
       "      <td>1900-11-14 12:00:12</td>\n",
       "      <td>1</td>\n",
       "      <td>1900-11-14</td>\n",
       "      <td>12</td>\n",
       "      <td>2</td>\n",
       "      <td>100</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>7.070797e+06</td>\n",
       "      <td>7.049394e+06</td>\n",
       "      <td>7.062005e+06</td>\n",
       "      <td>5979.578887</td>\n",
       "      <td>-0.596732</td>\n",
       "      <td>401</td>\n",
       "      <td>6.136033e+06</td>\n",
       "      <td>6.094996e+06</td>\n",
       "      <td>6.116389e+06</td>\n",
       "      <td>12055.148984</td>\n",
       "      <td>-0.331618</td>\n",
       "      <td>10.09</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.412219</td>\n",
       "      <td>2.496836</td>\n",
       "      <td>1.910336</td>\n",
       "      <td>359</td>\n",
       "      <td>0</td>\n",
       "      <td>139.067332</td>\n",
       "      <td>121.130025</td>\n",
       "      <td>0.372601</td>\n",
       "      <td>7.060750e+06</td>\n",
       "      <td>6.109599e+06</td>\n",
       "      <td>0.11</td>\n",
       "      <td>20.0</td>\n",
       "      <td>7.065627e+06</td>\n",
       "      <td>6.127272e+06</td>\n",
       "      <td>0.81</td>\n",
       "      <td>21402.484584</td>\n",
       "      <td>41036.883038</td>\n",
       "      <td>-913361.351922</td>\n",
       "      <td>9.758007e+05</td>\n",
       "      <td>1.917389</td>\n",
       "      <td>8.782913e+08</td>\n",
       "      <td>9.864835e+05</td>\n",
       "      <td>23</td>\n",
       "      <td>152911.229840</td>\n",
       "      <td>2095.387766</td>\n",
       "      <td>0.039900</td>\n",
       "      <td>0.089776</td>\n",
       "      <td>0.625</td>\n",
       "      <td>0.375</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.336658</td>\n",
       "      <td>0.231920</td>\n",
       "      <td>0.197007</td>\n",
       "      <td>0.234414</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ship             x           y_x   v_x  d_x                time  type  \\\n",
       "0     0  6.118352e+06  5.130672e+06  0.00    0 1900-11-07 12:09:28     0   \n",
       "1     1  6.102450e+06  5.112760e+06  0.00    0 1900-11-07 12:00:34     0   \n",
       "2     2  6.182482e+06  5.193696e+06  0.11  308 1900-11-14 12:07:01     0   \n",
       "3     3  5.228590e+06  4.606725e+06  1.67  211 1900-11-07 12:15:11     0   \n",
       "4     4  7.065534e+06  6.116614e+06  0.00  127 1900-11-14 12:00:12     1   \n",
       "\n",
       "         date  hour  weekday  v_cut  d_cut  hour_cut         x_max  \\\n",
       "0  1900-11-07    12        2    100      0         1  6.152038e+06   \n",
       "1  1900-11-07    12        2    100      0         1  6.102450e+06   \n",
       "2  1900-11-14    12        2      0      3         1  6.183191e+06   \n",
       "3  1900-11-07    12        2      1      2         1  5.287805e+06   \n",
       "4  1900-11-14    12        2    100      1         1  7.070797e+06   \n",
       "\n",
       "          x_min        x_mean         x_std    x_skew  x_count         y_max  \\\n",
       "0  6.118352e+06  6.119351e+06   5037.320747  5.255558      414  5.130781e+06   \n",
       "1  6.049472e+06  6.091460e+06  16543.394419 -1.058454      385  5.112874e+06   \n",
       "2  6.182482e+06  6.183011e+06    207.869601 -2.155218      233  5.193696e+06   \n",
       "3  5.228590e+06  5.239159e+06  17503.714347  1.608637      335  4.608628e+06   \n",
       "4  7.049394e+06  7.062005e+06   5979.578887 -0.596732      401  6.136033e+06   \n",
       "\n",
       "          y_min        y_mean         y_std    y_skew  v_max  v_min    v_mean  \\\n",
       "0  5.124873e+06  5.130494e+06    850.264541 -4.762308   9.39    0.0  0.265966   \n",
       "1  5.042857e+06  5.094050e+06  26764.042729 -0.802446  10.47    0.0  1.607922   \n",
       "2  5.193576e+06  5.193682e+06     21.740609 -4.563165  50.46    0.0  0.595150   \n",
       "3  4.577467e+06  4.601532e+06  11590.605179 -1.194210  10.09    0.0  1.471343   \n",
       "4  6.094996e+06  6.116389e+06  12055.148984 -0.331618  10.09    0.0  1.412219   \n",
       "\n",
       "      v_std     v_skew  d_max  d_min      d_mean       d_std    d_skew  \\\n",
       "0  1.321248   5.520205    129      0    4.613527   21.247770  4.483093   \n",
       "1  2.412688   1.590284    336      0   56.153247   91.449382  1.418867   \n",
       "2  3.415824  13.631590    360      0  123.356223  123.097127  0.657506   \n",
       "3  2.528593   2.135446    352      0  121.134328  121.758165  0.469794   \n",
       "4  2.496836   1.910336    359      0  139.067332  121.130025  0.372601   \n",
       "\n",
       "          x_q25           y_y   v_y   d_y         x_q75             y     v  \\\n",
       "0  6.118352e+06  5.130672e+06  0.00   0.0  6.118352e+06  5.130672e+06  0.00   \n",
       "1  6.074562e+06  5.061049e+06  0.00   0.0  6.102450e+06  5.112760e+06  3.45   \n",
       "2  6.183090e+06  5.193685e+06  0.11  10.0  6.183090e+06  5.193685e+06  0.32   \n",
       "3  5.229843e+06  4.595101e+06  0.11   0.0  5.236628e+06  4.608404e+06  2.21   \n",
       "4  7.060750e+06  6.109599e+06  0.11  20.0  7.065627e+06  6.127272e+06  0.81   \n",
       "\n",
       "    x_max_x_min   y_max_y_min    y_max_x_min   x_max_y_min     slope  \\\n",
       "0  33686.667453   5907.975523 -987570.399385  1.027165e+06  0.175380   \n",
       "1  52978.013345  70016.655842 -936597.872550  1.059593e+06  1.321617   \n",
       "2    708.835147    120.565000 -988786.086021  9.896155e+05  0.170089   \n",
       "3  59214.738740  31160.661097 -619962.107144  7.103375e+05  0.526232   \n",
       "4  21402.484584  41036.883038 -913361.351922  9.758007e+05  1.917389   \n",
       "\n",
       "           area      dis_mean  mode_hour   tot_distance    avg_speed  \\\n",
       "0  1.990200e+08  3.794369e+05         15   35968.022927   490.859961   \n",
       "1  3.709343e+09  4.245555e+05         19  190031.884257  2657.205033   \n",
       "2  8.546071e+04  2.899437e+05         17    3384.780162    62.731755   \n",
       "3  1.845170e+09  1.401766e+06         22  153492.071398  2641.442772   \n",
       "4  8.782913e+08  9.864835e+05         23  152911.229840  2095.387766   \n",
       "\n",
       "   v_cut_1_rate  v_cut_2_rate  v_hour_1_0  v_hour_1_1  v_hour_2_0  v_hour_2_1  \\\n",
       "0      0.002415      0.019324       0.000       1.000    0.000000    1.000000   \n",
       "1      0.005195      0.301299       0.000       1.000    0.508621    0.491379   \n",
       "2      0.017167      0.038627       0.500       0.500    0.555556    0.444444   \n",
       "3      0.044776      0.194030       0.600       0.400    0.723077    0.276923   \n",
       "4      0.039900      0.089776       0.625       0.375    0.500000    0.500000   \n",
       "\n",
       "   d_cut_0_rate  d_cut_1_rate  d_cut_2_rate  d_cut_3_rate  \n",
       "0      0.954106      0.045894      0.000000      0.000000  \n",
       "1      0.685714      0.127273      0.132468      0.054545  \n",
       "2      0.399142      0.248927      0.124464      0.227468  \n",
       "3      0.414925      0.217910      0.107463      0.259701  \n",
       "4      0.336658      0.231920      0.197007      0.234414  "
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_features.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_features.to_hdf('../train_features_all.h5' , 'df', mode='w')\n",
    "train_features = pd.read_hdf('../train_features_all.h5')\n",
    "# test_features.to_hdf('../test_features_all.h5' , 'df', mode='w')\n",
    "test_features = pd.read_hdf('../test_features_all.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###0.8796407970436798  == feat1， line score 0.873\n",
    "feat1 = \n",
    "39 x_mean,x_std,x_skew,x_count,y_max,y_min,y_mean,y_std,y_skew,v_max,v_mean,v_std,v_skew,d_mean,d_std,d_skew,x_q25,y_y,x_q75,x_max_x_min,y_max_y_min,y_max_x_min,x_max_y_min,slope,area,dis_mean,mode_hour,tot_distance,avg_speed,v_cut_1_rate,v_cut_2_rate,v_hour_1_0,v_hour_1_1,v_hour_2_0,v_hour_2_1,d_cut_0_rate,d_cut_1_rate,d_cut_2_rate,d_cut_3_rate\n",
    "\n",
    "['x_mean', 'x_std', 'x_skew', 'x_count', 'y_max', 'y_min', 'y_mean', 'y_std', 'y_skew', 'v_max', 'v_mean', 'v_std', 'v_skew', 'd_mean', 'd_std', 'd_skew', 'x_q25', 'y_y', 'x_q75', 'x_max_x_min', 'y_max_y_min', 'y_max_x_min', 'x_max_y_min', 'slope', 'area', 'dis_mean', 'mode_hour', 'tot_distance', 'avg_speed', 'v_cut_1_rate', 'v_cut_2_rate', 'v_hour_1_0', 'v_hour_1_1', 'v_hour_2_0', 'v_hour_2_1', 'd_cut_0_rate', 'd_cut_1_rate', 'd_cut_2_rate', 'd_cut_3_rate']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "39 x_mean,x_std,x_skew,x_count,y_max,y_min,y_mean,y_std,y_skew,v_max,v_mean,v_std,v_skew,d_mean,d_std,d_skew,x_q25,y_y,x_q75,x_max_x_min,y_max_y_min,y_max_x_min,x_max_y_min,slope,area,dis_mean,mode_hour,tot_distance,avg_speed,v_cut_1_rate,v_cut_2_rate,v_hour_1_0,v_hour_1_1,v_hour_2_0,v_hour_2_1,d_cut_0_rate,d_cut_1_rate,d_cut_2_rate,d_cut_3_rate\n"
     ]
    }
   ],
   "source": [
    "common_drop = ['ship', 'x', 'y', 'v', 'd', 'time', 'type', 'date', 'hour', 'weekday','v_cut', 'd_cut', 'hour_cut']\n",
    "del_col1 = ['dis_x_mean'] +['d_max', 'd_y', 'v_y', 'v_min', 'd_min', 'x_diff', 'y_diff', 'time_diff', 'ind_speed', 'ind_dist','y_x','v_x','d_x','x_max','x_min']  \n",
    "#,'y_y'\n",
    "features = [x for x in train_features.columns if x not in  common_drop+del_col1 ]\n",
    "target = 'type'\n",
    "print(len(features), ','.join(features))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 加入队友特征\n",
    "train_feat = pd.read_csv('../trn_per_300.csv')\n",
    "test_feat = pd.read_csv('../tst_per_300.csv')\n",
    "\n",
    "train_features_team = train_features.merge(train_feat.drop('type', axis =1), how ='left' ,on ='ship')\n",
    "test_features_team = test_features.merge(test_feat, how ='left' ,on ='ship')\n",
    "\n",
    "common_drop = ['ship', 'x', 'y', 'v', 'd', 'time', 'type', 'date', 'hour', 'weekday','v_cut', 'd_cut', 'hour_cut']\n",
    "del_col1 = ['dis_x_mean'] +['d_max', 'd_y', 'v_y', 'v_min', 'd_min', 'x_diff', 'y_diff', 'time_diff', 'ind_speed', 'ind_dist','y_x','v_x','d_x','x_max','x_min']  \n",
    "#,'y_y'\n",
    "features = [x for x in train_features_team.columns if x not in  common_drop+del_col1 ]\n",
    "target = 'type'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2000, 1260)"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_features_team.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_features = train_features_team.copy()\n",
    "test_features = test_features_team.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "    'n_estimators': 5000,\n",
    "    'boosting_type': 'gbdt',\n",
    "    'objective': 'multiclass',\n",
    "    'num_class': 3,\n",
    "    'early_stopping_rounds': 100,\n",
    "}\n",
    "\n",
    "# params = {\n",
    "#     'n_estimators': 3000,\n",
    "#     'boosting_type': 'gbdt',    \n",
    "#     'learning_rate': 0.01,\n",
    "#     'lambda_l1': 0.0,\n",
    "#     'lambda_l2': 0.01,\n",
    "#     'max_depth': 7,\n",
    "#     'objective': 'multiclass',  # 目标函数\n",
    "#     'num_class': 3,\n",
    "#     'early_stopping_rounds': 100,\n",
    "# } 线下表现0.89，但线上表现不好  0.8706"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\ttraining's multi_logloss: 0.0613298\tvalid_1's multi_logloss: 0.244144\n",
      "[200]\ttraining's multi_logloss: 0.0130816\tvalid_1's multi_logloss: 0.232239\n",
      "Early stopping, best iteration is:\n",
      "[169]\ttraining's multi_logloss: 0.0209188\tvalid_1's multi_logloss: 0.231369\n",
      "0 val f1 0.8905830266074947\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\ttraining's multi_logloss: 0.0615836\tvalid_1's multi_logloss: 0.22696\n",
      "[200]\ttraining's multi_logloss: 0.0129216\tvalid_1's multi_logloss: 0.225598\n",
      "Early stopping, best iteration is:\n",
      "[142]\ttraining's multi_logloss: 0.0314011\tvalid_1's multi_logloss: 0.221876\n",
      "1 val f1 0.8996347786256567\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\ttraining's multi_logloss: 0.0628683\tvalid_1's multi_logloss: 0.217517\n",
      "[200]\ttraining's multi_logloss: 0.0137217\tvalid_1's multi_logloss: 0.204231\n",
      "Early stopping, best iteration is:\n",
      "[159]\ttraining's multi_logloss: 0.0252206\tvalid_1's multi_logloss: 0.201389\n",
      "2 val f1 0.889886581983241\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\ttraining's multi_logloss: 0.057806\tvalid_1's multi_logloss: 0.257691\n",
      "[200]\ttraining's multi_logloss: 0.0119618\tvalid_1's multi_logloss: 0.259769\n",
      "Early stopping, best iteration is:\n",
      "[128]\ttraining's multi_logloss: 0.0366889\tvalid_1's multi_logloss: 0.251934\n",
      "3 val f1 0.8628107444464833\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\ttraining's multi_logloss: 0.0609765\tvalid_1's multi_logloss: 0.23677\n",
      "[200]\ttraining's multi_logloss: 0.0129147\tvalid_1's multi_logloss: 0.236146\n",
      "Early stopping, best iteration is:\n",
      "[147]\ttraining's multi_logloss: 0.0291818\tvalid_1's multi_logloss: 0.229466\n",
      "4 val f1 0.8847333046851329\n"
     ]
    }
   ],
   "source": [
    "fold = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n",
    "\n",
    "X = train_features[features].copy()\n",
    "y = train_features[target]\n",
    "models = []\n",
    "pred = np.zeros((len(test_label),3))\n",
    "oof = np.zeros((len(X), 3))\n",
    "\n",
    "for index, (train_idx, val_idx) in enumerate(fold.split(X, y)):\n",
    "\n",
    "    train_set = lgb.Dataset(X.iloc[train_idx], y.iloc[train_idx])\n",
    "    val_set = lgb.Dataset(X.iloc[val_idx], y.iloc[val_idx])\n",
    "\n",
    "    model = lgb.train(params, train_set, valid_sets=[train_set, val_set], verbose_eval=100)\n",
    "    models.append(model)\n",
    "    val_pred = model.predict(X.iloc[val_idx])\n",
    "    oof[val_idx] = val_pred\n",
    "    val_y = y.iloc[val_idx]\n",
    "    val_pred = np.argmax(val_pred, axis=1)\n",
    "    print(index, 'val f1', metrics.f1_score(val_y, val_pred, average='macro'))\n",
    "\n",
    "    test_pred = model.predict(test_features[features])\n",
    "    pred += test_pred/5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "oof f1 0.8912259549855474\n"
     ]
    }
   ],
   "source": [
    "oof = np.argmax(oof, axis=1)\n",
    "best_off = metrics.f1_score(oof, y, average='macro')\n",
    "print('oof f1', best_off)\n",
    "# 20200207  0.8698375593266509   0.8708332303537429  0.8726318869413454  0.8785484732295895 \n",
    "# 0.8842927996963574（drop50） 0.8835615929288121（drop 70） 0.8898887628114594 （drop 72） 0.887897628114594 （drop 75）\n",
    "# 20200210 0.8768242029333674\n",
    "# 20200211   0.8796407970436798"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    0.6315\n",
      "1    0.2330\n",
      "2    0.1355\n",
      "Name: pred, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "pred = np.argmax(pred, axis=1)\n",
    "sub = test_features[['ship']]\n",
    "sub['pred'] = pred\n",
    "\n",
    "print(sub['pred'].value_counts(1))\n",
    "sub['pred'] = sub['pred'].map(type_map_rev)\n",
    "dt = datetime.today().strftime('%y%m%d') \n",
    "sub.to_csv('result_%s_%s.csv' % (dt,best_off), index=None, header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "ret = []\n",
    "for index, model in enumerate(models):\n",
    "    df = pd.DataFrame()\n",
    "    df['name'] = model.feature_name()\n",
    "    df['score'] = model.feature_importance()\n",
    "    df['fold'] = index\n",
    "    ret.append(df)\n",
    "    \n",
    "df = pd.concat(ret)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.groupby('name', as_index=False)['score'].mean()\n",
    "df = df.sort_values(['score'], ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfData = train_features[features].corr()\n",
    "dfData = dfData[df.head(15)['name'].to_list()]\n",
    "plt.subplots(figsize=(16, 16))\n",
    "sns.heatmap(dfData)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
